In [1]:
# imports
import os
os.sys.path.append(os.path.abspath(".."))
os.sys.path.append(os.path.abspath("../.."))
import tensorflow as tf
import pandas as pd
import numpy as np
import h5py
from tqdm import tqdm
from matplotlib import pyplot as plt
from plotly import graph_objects as go
import plotly.express as px
from project.notebooks_utils import visuals
import project.download_content as content
from project.utils import data
from project.utils import data_augmentation as da
from project.model.loss import BBOX_REF
from project import train
from copy import deepcopy
import project.download_content as content
from keras.preprocessing import image
from keras.optimizers import Adam, SGD
import pickle
from project.utils.data_bbox_match_hdf5 import load_ohc
datapath = os.path.join(content.DATAPATH, "MODEL", "data_300_vgg.h5")
anchors = BBOX_REF.references.values
tqdm.pandas()
In [2]:
# loading data
images, X = da.load_data('1')
# chosen one image
num = 8121
id, path = images[num]
path = '../' + path
bboxes_raw = X[num]
def plt_img(img):
image_bin = image.array_to_img(img)
plt.figure(figsize=(8, 6))
plt.imshow(image_bin)
def plt_bboxes(img, bboxes):
image_bin = image.array_to_img(img)
visuals.draw_many_bboxes(image_bin, bboxes[:, -4:], label='ref')
refs = [anchors[i] for i in BBOX_REF.match(bboxes[:, -4:])]
visuals.draw_many_bboxes(image_bin, refs, label='anc', color="#ff0000");
Image without change¶
In [3]:
img_bin = image.load_img(path, target_size=(300, 300))
img = image.img_to_array(img_bin)
plt_img(img)
plt_bboxes(img, bboxes_raw)
flip horizontaly¶
In [4]:
bboxes = deepcopy(bboxes_raw)
img_flip_h = np.flip(img, 1)
bboxes[:, -4] = 1 - bboxes[:, -4]
plt_img(img_flip_h)
plt_bboxes(img, bboxes)
flip vertically¶
In [5]:
bboxes = deepcopy(bboxes_raw)
img_flip_v = np.flip(img, 0)
bboxes[:, -3] = 1 - bboxes[:, -3]
plt_img(img_flip_v)
plt_bboxes(img, bboxes)
flip horizontaly and vertically¶
In [6]:
bboxes = deepcopy(bboxes_raw)
img_flip_h_v = np.flip(img, [0, 1])
bboxes[:, -4:-2] = 1 - bboxes[:, -4:-2]
plt_img(img_flip_h_v)
plt_bboxes(img, bboxes)
zoom 1¶
In [7]:
bboxes = deepcopy(bboxes_raw)
img_z, bboxes = da.resize(img, bboxes, .8, .09, .09)
plt_img(np.array(img_z))
plt_bboxes(img_z, bboxes)
zoom 2¶
In [8]:
bboxes = deepcopy(bboxes_raw)
img_z, bboxes = da.resize(img, bboxes, .42, -.05, .25)
plt_img(np.array(img_z))
plt_bboxes(img_z, bboxes)
Original (again)¶
In [9]:
plt_img(img)
Saturation¶
In [10]:
img_bright = tf.image.random_saturation(img, .4, .45).numpy()
plt_img(img_bright)
Contrast¶
In [11]:
img_bright = tf.image.random_contrast(img, .5, .6).numpy()
plt_img(img_bright)
Data Augmentationa criteria¶
In [12]:
classes_count_path = os.path.join(content.DATAPATH, 'METADATA',
'train_classes_count_bboxes.csv')
df_count_train = pd.read_csv(classes_count_path)
display(df_count_train[:30])
The following graphs explain the rational, but the main concept is based on how unbalanced is the dataset. So, the more irrelevant the category is, the more data augmentation I have to do.
I divided data augmentation in six rules:
- rule 0: original image only (no augmentation)
- rule 1: rule 0 + horizontal flip
- rule 2: rule 1 + saturation
- rule 3: rule 2 + contrast + horizontal flip with saturation
- rule 4: rule 3 + zoom 80% (get 80% of the image, lesser the number greater the zoom)
- rule 5: rule 4 + zoom 60% (get 60% of the image, lesser the number greater the zoom)
In [13]:
df_count_train['DataAug'] = 1
df_count = df_count_train[['LabelSemantic', 'Count', 'DataAug']]
df_count['data'] = 'raw'
df_aug = df_count.copy()
df_aug['data'] = 'augmented'
def incremental_rule(x):
level = 100000/x
if level < 2:
# 1st rule: original image + horizontal_flip
return 2
if level < 4:
# 2nd rule: 1st rule + saturation
return 3
if level < 10:
# 3rd rule: 2nd rule + contrast + flip_with_saturation
return 5
elif level < 30:
# 4th rule: 3rd rule + zoom_80
return 10
else:
# 5th rule: 4th rule + zoom_60
return 13
indexes = df_aug[df_aug.Count < 100000].index
df_aug.loc[indexes, 'DataAug'] = df_aug.loc[indexes].Count.apply(incremental_rule)
df_aug.loc[indexes, 'Count'] = df_aug['DataAug'] * df_aug['Count']
fig = go.Figure([go.Scatter(x=df_aug.LabelSemantic.values,
y=df_aug.Count.values,
marker_color='#d97038',
fill='tonexty',
name='augmented data'),
go.Scatter(x=df_count.LabelSemantic.values,
y=df_count.Count.values,
marker_color='#005191',
fill='tozeroy',
name='raw data')])
fig.update_layout(barmode='stack', yaxis_type='log',
title_text='Amount of expected increased data with data augumentation')
fig.show()
# plotting after the 15th category - trying to avoid outliers categories in violin plot
df_all = pd.concat([df_count.iloc[15:], df_aug.iloc[15:]]).reset_index()[['LabelSemantic', 'Count', 'data']]
fig = px.violin(df_all, y="Count", color="data", points='all', color_discrete_sequence=['#005191', '#d97038'])
fig.update_layout(title_text='Data distributions by type')
fig.show()
find category ids in one hot encode model¶
In [14]:
ohc = load_ohc()
ohc_classes = [c[3:] for c in ohc.get_feature_names()]
def id_in_ohc(x):
x = x.replace(' ', '_').lower()
return ohc_classes.index(x)
df_aug['OHC'] = df_aug.LabelSemantic.apply(id_in_ohc)
In [15]:
print(f'Categories to do not augment\n',
df_aug[df_aug.DataAug == 1].OHC.tolist(), end='\n\n')
print(f'Categories to 1st level augmentation\n',
df_aug[df_aug.DataAug == 2].OHC.tolist(), end='\n\n')
print(f'Categories to 2nd level augmentation\n',
df_aug[df_aug.DataAug == 3].OHC.tolist(), end='\n\n')
print(f'Categories to 3rd level augmentation\n',
df_aug[df_aug.DataAug == 5].OHC.tolist(), end='\n\n')
print(f'Categories to 4th level augmentation\n',
df_aug[df_aug.DataAug == 10].OHC.tolist(), end='\n\n')
print(f'Categories to 5th level augmentation\n',
df_aug[df_aug.DataAug == 13].OHC.tolist(), end='\n\n')
In [16]:
data_aug_criteria_path = os.path.join(content.DATAPATH,
'data_aug.json')
def aug_level(x):
if x == 1:
return 0
elif x == 2:
return 1
elif x == 3:
return 2
elif x == 5:
return 3
elif x == 10:
return 4
elif x == 13:
return 5
else:
raise RuntimeError('Invalid data aug!')
df_aug['AugLevel'] = df_aug.DataAug.apply(aug_level)
df_aug['Label'] = df_aug.LabelSemantic.apply(lambda x: x.replace(' ', '_').lower())
(pd.DataFrame(df_aug.groupby('AugLevel')['OHC'].apply(list))
.join(pd.DataFrame(df_aug.groupby('AugLevel')['Label'].apply(list)))
.to_json(data_aug_criteria_path, orient='index'))